package com.imooc.structured.streaming

import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.types.{IntegerType, StringType, StructField, StructType}

/**
  * http://spark.apache.org/docs/latest/structured-streaming-programming-guide.html#input-sources
  *
  * @description Structured Streaming 处理File数据
  * @author yuyon26@126.com
  * @date 2018/10/27 15:13
  */
object StructuredFileWordCount {
  def main(args: Array[String]): Unit = {

    val spark = SparkSession
      .builder
      .appName("StructuredNetworkWordCount")
      .master("local[2]")
      .getOrCreate()
    spark.sparkContext.setLogLevel("ERROR")

    import spark.implicits._
    //创建一个DataFrame
    val userSchema = StructType(
      Array(
        StructField("name", StringType),
        StructField("age", StringType),
        StructField("job", StringType)
      )
    )
    val linesDataFrames = spark.readStream
      .option("sep", ";")
      .schema(userSchema)
      .csv("D:\\spark\\spark_streaming\\csv")

    val query = linesDataFrames.writeStream
      .format("console")
      .start()

    query.awaitTermination()
  }
}
